package com.splider.toutiao;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;

import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;

public class TTHaoSplider {


	// 头条号主页地址
	public static final String TOUTIAOHAOURL = "http://www.toutiao.com/pgc/ma/?page_type=1&count=10&version=2&platform=pc";
	
	// 头条号主页地址
	public static final String TOUTIAOHAOURL1 = "http://www.toutiao.com/c/user/article/?page_type=1&count=20";

	private static Map<String, String> header = new HashMap<String, String>();
	
	private static Map<String, String> header1 = new HashMap<String, String>();

	public static void main(String[] args) {
		
		//设置请求头
		header.put("Host", "www.toutiao.com");
		header.put("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0");
		header.put("Accept","text/javascript, text/html, application/xml, text/xml, */*");
		header.put("Accept-Language","zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3");
		header.put("Accept-Encoding","gzip, deflate");
		header.put("X-Requested-With","XMLHttpRequest");
		header.put("Content-Type","text/*, application/xml");
		header.put("Connection","keep-alive");
		header.put("Cookie", "_ga=GA1.2.1579840638.1479797898; uuid=\"w:1c8b3d677a314defa8b63310a32235e9\"; tt_webid=36837665425; CNZZDATA1259612802=141991597-1479797317-%7C1484033888; csrftoken=58b7438e8257c7699bf7032e0de66ce3; __tasessionId=6s9gr2ktw1484035796735");
		
		
		header1.put("Host", "www.toutiao.com");
		header1.put("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.22 Safari/537.36 SE 2.X MetaSr 1.0");
		header1.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
		header1.put("Accept-Language","zh-CN,zh;q=0.8");
		header1.put("Accept-Encoding","gzip, deflate, sdch");
		header1.put("Content-Type","application/octet-stream");
		header1.put("Connection","keep-alive");
		header1.put("Upgrade-Insecure-Requests", "1");
		header1.put("Cookie", "CNZZDATA1258609184=1974985307-1467773945-null%7C1467773945; csrftoken=e4c83e1812298f85f8e4078dc8b57218; CNZZDATA1259612802=1377273964-1479092337-%7C1480321095; BDTUJIAID=7eabb1c9ec08aff42d58877b65078234; uuid=\"w:bb7ef44dc245484abdc44f6d87ea2ce5\"; install_id=0; tt_webid=52135437746; _ba=BA0.2-20170216-51d9e-LDtGd5em3NsepU2fM7sV; utm_source=toutiao; login_flag=603070cdb797d6bc8f92bd826a5d84a1; sessionid=e7b197b90996ef5572160140f4008667; sid_tt=e7b197b90996ef5572160140f4008667; sid_guard=\"e7b197b90996ef5572160140f4008667|1487235417|2592000|Sat\054 18-Mar-2017 08:56:57 GMT\"; tadseekerHwuid=01934733383571565624160144754383; csrftoken=64cf6d84b869e5dfcb0be5b2474d91b3; __utmt=1; __utma=24953151.1423919142.1473230525.1483683326.1487236733.13; __utmb=24953151.3.10.1487236733; __utmc=24953151; __utmz=24953151.1483683326.12.5.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; cp=58A537E6540F4E1; __tasessionId=w09txf4jg1487235416093; CNZZDATA1259612802=1377273964-1479092337-%7C1487233765; _ga=GA1.2.2132494613.1484701428");
		long time = 0;
		String haoId = "3365634632";
		for (; time >= 0;) {
		   time = connectHao(haoId,time);
		   try {
			  Thread.sleep(5000);
		   } catch (InterruptedException e) {
		   }
		}
	}
	
	

	/**
	 * 根据头条号的id获取该号的文章
	 * 
	 * @param haoId    头条号id
	 * @param hotTime  下拉刷新时获取下一个新闻队列
	 * @return
	 */
	private static long connectHao(String haoId, long hotTime) {
		
		String haoUrl = "/c/user/"+haoId+"/";
		String media_id = haoUrl.replace("http://toutiao.com/m", "");
		media_id = media_id.replace("/", "");
		media_id = media_id.replace("m", "");
		//System.out.println("media_id = " + media_id);
		header1.put("Referer", "http://toutiao.com/m"+media_id);
		JSONObject json = getTopicList(media_id, hotTime);
		
		System.out.println("头条号  = " + haoId + "  || hotTime = " +hotTime + "  || json = "+json);
		if (json != null) {
			long time = json.getJSONObject("next").getLongValue("max_behot_time");
			JSONArray data = json.getJSONArray("data");
			int addCount = 0;
			for (int i = 0; i < data.size(); i++) {
				try {
					JSONObject obj = (JSONObject) data.get(i);
					String articleUrl = obj.getString("source_url");
					String title = obj.getString("title");// 标题
					String articleId = articleUrl.substring(articleUrl.indexOf("item/") + 5);
					articleId = articleId.replace("/", "");
					if (articleId.matches("[0-9]+")) {
									
						// 排除视频和图册
						if (obj.getBooleanValue("has_video") || obj.getBooleanValue("has_gallery")) {
							continue;
						}
						System.out.println("title = " + title + " || articleUrl = " + articleUrl);
						Map<String, Object> articleInfoMap = getArticleInfo(articleUrl);
						System.out.println(articleInfoMap.get("contentText"));
						Thread.sleep(1000);
					//	System.err.println("######################################################################################################");
					
					}
				} catch (Exception e) {
					e.printStackTrace();
					System.out.println("error:" + e.getMessage());
				}
			}
			if(addCount>0){
				System.out.println("读取更新:" + addCount + "条");
			}
			if(json.getIntValue("has_more") == 1){
				return time;
			}
		} else {
			System.out.println("该头条号读取完成,max_behot_time:" + hotTime);
		}
		return -1;
	}
	
    /**
     * 
     * @param media_id
     * @param hotTime
     * @return
     */
	public static JSONObject getTopicList(String media_id, long hotTime) {
		
		String url = TOUTIAOHAOURL + "&media_id=" + media_id + "&max_behot_time=" + hotTime;
		if(media_id.contains("user")){
			url = TOUTIAOHAOURL1 + "&user_id="+media_id.replace("cuser", "")+"&max_behot_time="+hotTime;
		}
		JSONObject param = null;
		param = CommonUtils.getUrlParam();
		url += "&as=" + param.get("as") + "&cp=" + param.get("cp");
		System.out.println(url);
		try {
			URL httpUrl = new URL(url);
			httpUrl.openConnection().setConnectTimeout(Integer.parseInt(ApplicationConfig.prop.getProperty("time_out")));
			httpUrl.openConnection().setReadTimeout(Integer.parseInt(ApplicationConfig.prop.getProperty("time_out")));
			BufferedReader in = new BufferedReader(new InputStreamReader(httpUrl.openStream(), "UTF-8"));
			String line = null;
			StringBuffer content = new StringBuffer("");
			while ((line = in.readLine()) != null) {
				content.append(line);
			}
			in.close();
			line = null;
			in = null;
			httpUrl = null;
			return JSONObject.parseObject(content.toString());
		} catch (Exception e) {
           e.printStackTrace();
		}
		return null;
	}

	/**
	 * 根据文章url获取该文章详情页
	 * @param url
	 * @return
	 */
	public static Map<String, Object> getArticleInfo(String url) {
		if(!url.contains("toutiao.com")){
			url = "http://www.toutiao.com"+url;
		}
		try {
			Map<String, Object> map = new HashMap<String, Object>();
			Connection connect = Jsoup.connect(url).headers(header1);
			Document document;
			document = connect.timeout(Integer.parseInt(ApplicationConfig.prop.getProperty("time_out"))).get();
			
			return map;
		} catch (IOException e) {
			System.out.println("访问文章页失败:" + url + "  原因" + e.getMessage());
		}
		return null;
	}
	
	
}
